import blogcrawlerUtil;
import UrlDb;
import CrawlerConfig;
from OutputDataParser import ProfileFile;
from OutputDataParser import PostingFile;

import os;
import time;
import threading;

class ProfileCrawler:
    def __init__(self, urlConverter, profileParser, urlFilter, config):
        self.urlConverter = urlConverter;
        self.profileParser = profileParser;
        self.urlFilter = urlFilter;
        
        self.toCrawlHomeUrlFilePath = config.toCrawlHomeUrlFilePath;
        self.toCrawlHomeUrlFileLock = config.toCrawlHomeUrlFileLock;
        
        taskId = config.taskId;
        self.homeUrlFilePath = os.path.join(taskId, 'home.url.dat');
        self.homeUrlFileLock = threading.Lock();
        
        self.homeUrlDb = self.recoverDb(self.homeUrlFilePath, os.path.join(taskId, 'home.url.db'));
        self.homeUrlDbLock = threading.Lock();
        
        self.profileFilePath = os.path.join(taskId, 'profile.dat');
        self.profileFileLock = threading.Lock();
        
        self.tempFilePath = os.path.join(taskId , '.profile.tmp');
            
    def recoverDb(self, urlFilePath, urlDbPath):
        if os.path.exists(urlDbPath):
            os.remove(urlDbPath);
        urlDb = UrlDb.UrlDb(urlDbPath);
        if os.path.exists(urlFilePath):
            urlFile = open(urlFilePath);
            line = urlFile.readline();
            while line:
                url = line.strip();
                urlDb.add(url);
                line = urlFile.readline();
        return urlDb;
    
     
    def crawl(self):
        while 1:
            urls = self.getHomeUrls();
            self.crawlProfiles(urls);
            if len(urls) == 0:
                time.sleep(120);
            
    def crawlProfiles(self, urls):
        cnt = 0;
        for homeUrl in urls:
            self.homeUrlDbLock.acquire();
            isContains = self.homeUrlDb.contains(homeUrl);
            self.homeUrlDbLock.release();
            if not isContains:
                profileUrl = self.urlConverter.home2Profile(homeUrl);
                profile = self.getProfile(profileUrl);
                if profile:
                    self.homeUrlDbLock.acquire();
                    if self.homeUrlDb.contains(homeUrl):
                        self.homeUrlDbLock.release();
                    else:                       
                        self.homeUrlDb.add(homeUrl);
                        self.homeUrlDbLock.release();
                        self.homeUrlFileLock.acquire();
                        homeUrlFile = open(self.homeUrlFilePath, 'a');
                        homeUrlFile.write(profile.url + '\n');
                        homeUrlFile.close();
                        self.homeUrlFileLock.release();
                        self.profileFileLock.acquire();
                        profileFile = open(self.profileFilePath, 'a');
                        profileFile.write(profile.__str__() + '\n');
                        profileFile.flush();
                        profileFile.close();
                        self.profileFileLock.release();
                        cnt += 1;
        print time.asctime(), ': adding',cnt,'profile';
        
    def getHomeUrls(self):
        if not os.path.exists(self.toCrawlHomeUrlFilePath):
            return [];
        self.toCrawlHomeUrlFileLock.acquire();
        toCrawlHomeUrlFile = open(self.toCrawlHomeUrlFilePath);
        lines = toCrawlHomeUrlFile.readlines();
        toCrawlHomeUrlFile.close();
        os.remove(self.toCrawlHomeUrlFilePath);
        self.toCrawlHomeUrlFileLock.release();
        
        urlSet = set();
        for line in lines:
            url = line.strip();
            if self.urlFilter.isValid(url):
                urlSet.add(url);
        urlList = list(urlSet);
        print 'ProfileCrawler::getHomeUrl(): get', len(urlList), 'valid home url!'
        return urlList;
    
    def getProfile(self, profileUrl):
        blogcrawlerUtil.crawlPage(self.tempFilePath, profileUrl);
        try:
            profile = self.profileParser.parse(self.tempFilePath);
        except:
            print 'ERROR in parsing profile url \"' + profileUrl + '\"';
            return '';
        profile.url = self.urlConverter.profile2Home(profileUrl);
        os.remove(self.tempFilePath);
        return profile;    
        